/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package wmr.util;

import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import org.apache.hadoop.io.Text;



/**
 *
 * @author shilad
 */
public class Utils {

	public static final String[] STOP_WORDS = {
		"a's",
		"able",
		"about",
		"above",
		"according",
		"accordingly",
		"across",
		"actually",
		"after",
		"afterwards",
		"again",
		"against",
		"ain't",
		"all",
		"allow",
		"allows",
		"almost",
		"alone",
		"along",
		"already",
		"also",
		"although",
		"always",
		"am",
		"among",
		"amongst",
		"an",
		"and",
		"another",
		"any",
		"anybody",
		"anyhow",
		"anyone",
		"anything",
		"anyway",
		"anyways",
		"anywhere",
		"apart",
		"appear",
		"appreciate",
		"appropriate",
		"are",
		"aren't",
		"around",
		"as",
		"aside",
		"ask",
		"asking",
		"associated",
		"at",
		"available",
		"away",
		"awfully",
		"be",
		"became",
		"because",
		"become",
		"becomes",
		"becoming",
		"been",
		"before",
		"beforehand",
		"behind",
		"being",
		"believe",
		"below",
		"beside",
		"besides",
		"best",
		"better",
		"between",
		"beyond",
		"both",
		"brief",
		"but",
		"by",
		"c'mon",
		"c's",
		"came",
		"can",
		"can't",
		"cannot",
		"cant",
		"cause",
		"causes",
		"certain",
		"certainly",
		"changes",
		"clearly",
		"co",
		"com",
		"come",
		"comes",
		"concerning",
		"consequently",
		"consider",
		"considering",
		"contain",
		"containing",
		"contains",
		"corresponding",
		"could",
		"couldn't",
		"course",
		"currently",
		"definitely",
		"described",
		"despite",
		"did",
		"didn't",
		"different",
		"do",
		"does",
		"doesn't",
		"doing",
		"don't",
		"done",
		"down",
		"downwards",
		"during",
		"each",
		"edu",
		"eg",
		"eight",
		"either",
		"else",
		"elsewhere",
		"enough",
		"entirely",
		"especially",
		"et",
		"etc",
		"even",
		"ever",
		"every",
		"everybody",
		"everyone",
		"everything",
		"everywhere",
		"ex",
		"exactly",
		"example",
		"except",
		"far",
		"few",
		"fifth",
		"first",
		"five",
		"followed",
		"following",
		"follows",
		"for",
		"former",
		"formerly",
		"forth",
		"four",
		"from",
		"further",
		"furthermore",
		"get",
		"gets",
		"getting",
		"given",
		"gives",
		"go",
		"goes",
		"going",
		"gone",
		"got",
		"gotten",
		"greetings",
		"had",
		"hadn't",
		"happens",
		"hardly",
		"has",
		"hasn't",
		"have",
		"haven't",
		"having",
		"he",
		"he's",
		"hello",
		"help",
		"hence",
		"her",
		"here",
		"here's",
		"hereafter",
		"hereby",
		"herein",
		"hereupon",
		"hers",
		"herself",
		"hi",
		"him",
		"himself",
		"his",
		"hither",
		"hopefully",
		"how",
		"howbeit",
		"however",
		"i'd",
		"i'll",
		"i'm",
		"i've",
		"ie",
		"if",
		"ignored",
		"immediate",
		"in",
		"inasmuch",
		"inc",
		"indeed",
		"indicate",
		"indicated",
		"indicates",
		"inner",
		"insofar",
		"instead",
		"into",
		"inward",
		"is",
		"isn't",
		"it",
		"it'd",
		"it'll",
		"it's",
		"its",
		"itself",
		"just",
		"keep",
		"keeps",
		"kept",
		"know",
		"knows",
		"known",
		"last",
		"lately",
		"later",
		"latter",
		"latterly",
		"least",
		"less",
		"lest",
		"let",
		"let's",
		"like",
		"liked",
		"likely",
		"little",
		"look",
		"looking",
		"looks",
		"ltd",
		"mainly",
		"many",
		"may",
		"maybe",
		"me",
		"mean",
		"meanwhile",
		"merely",
		"might",
		"more",
		"moreover",
		"most",
		"mostly",
		"much",
		"must",
		"my",
		"myself",
		"name",
		"namely",
		"nd",
		"near",
		"nearly",
		"necessary",
		"need",
		"needs",
		"neither",
		"never",
		"nevertheless",
		"new",
		"next",
		"nine",
		"no",
		"nobody",
		"non",
		"none",
		"noone",
		"nor",
		"normally",
		"not",
		"nothing",
		"novel",
		"now",
		"nowhere",
		"obviously",
		"of",
		"off",
		"often",
		"oh",
		"ok",
		"okay",
		"old",
		"on",
		"once",
		"one",
		"ones",
		"only",
		"onto",
		"or",
		"other",
		"others",
		"otherwise",
		"ought",
		"our",
		"ours",
		"ourselves",
		"out",
		"outside",
		"over",
		"overall",
		"own",
		"particular",
		"particularly",
		"per",
		"perhaps",
		"placed",
		"please",
		"plus",
		"possible",
		"presumably",
		"probably",
		"provides",
		"que",
		"quite",
		"qv",
		"rather",
		"rd",
		"re",
		"really",
		"reasonably",
		"regarding",
		"regardless",
		"regards",
		"relatively",
		"respectively",
		"right",
		"said",
		"same",
		"saw",
		"say",
		"saying",
		"says",
		"second",
		"secondly",
		"see",
		"seeing",
		"seem",
		"seemed",
		"seeming",
		"seems",
		"seen",
		"self",
		"selves",
		"sensible",
		"sent",
		"serious",
		"seriously",
		"seven",
		"several",
		"shall",
		"she",
		"should",
		"shouldn't",
		"since",
		"six",
		"so",
		"some",
		"somebody",
		"somehow",
		"someone",
		"something",
		"sometime",
		"sometimes",
		"somewhat",
		"somewhere",
		"soon",
		"sorry",
		"specified",
		"specify",
		"specifying",
		"still",
		"sub",
		"such",
		"sup",
		"sure",
		"t's",
		"take",
		"taken",
		"tell",
		"tends",
		"th",
		"than",
		"thank",
		"thanks",
		"thanx",
		"that",
		"that's",
		"thats",
		"the",
		"their",
		"theirs",
		"them",
		"themselves",
		"then",
		"thence",
		"there",
		"there's",
		"thereafter",
		"thereby",
		"therefore",
		"therein",
		"theres",
		"thereupon",
		"these",
		"they",
		"they'd",
		"they'll",
		"they're",
		"they've",
		"think",
		"third",
		"this",
		"thorough",
		"thoroughly",
		"those",
		"though",
		"three",
		"through",
		"throughout",
		"thru",
		"thus",
		"to",
		"together",
		"too",
		"took",
		"toward",
		"towards",
		"tried",
		"tries",
		"truly",
		"try",
		"trying",
		"twice",
		"two",
		"un",
		"under",
		"unfortunately",
		"unless",
		"unlikely",
		"until",
		"unto",
		"up",
		"upon",
		"us",
		"use",
		"used",
		"useful",
		"uses",
		"using",
		"usually",
		"value",
		"various",
		"very",
		"via",
		"viz",
		"vs",
		"want",
		"wants",
		"was",
		"wasn't",
		"way",
		"we",
		"we'd",
		"we'll",
		"we're",
		"we've",
		"welcome",
		"well",
		"went",
		"were",
		"weren't",
		"what",
		"what's",
		"whatever",
		"when",
		"whence",
		"whenever",
		"where",
		"where's",
		"whereafter",
		"whereas",
		"whereby",
		"wherein",
		"whereupon",
		"wherever",
		"whether",
		"which",
		"while",
		"whither",
		"who",
		"who's",
		"whoever",
		"whole",
		"whom",
		"whose",
		"why",
		"will",
		"willing",
		"wish",
		"with",
		"within",
		"without",
		"won't",
		"wonder",
		"would",
		"would",
		"wouldn't",
		"yes",
		"yet",
		"you",
		"you'd",
		"you'll",
		"you're",
		"you've",
		"your",
		"yours",
		"yourself",
		"yourselves",
		"zero"
	};
    public static final Set<String> STOP_WORDS_SET;
    static {
        STOP_WORDS_SET = new HashSet<String>();
        for (String w : STOP_WORDS) {
        	STOP_WORDS_SET.add(w);
        }
    }


    public static String stem(String word) {
        Stemmer s = new Stemmer();
        s.add(word.toCharArray(), word.length());
        s.stem();
        return s.toString();
    }

    public static String truncateDouble(String s, int n) {
        if (s.length() <= n) {
            return s;
        }
        int i = s.indexOf("E");
        if (i >= 0) {
            return s.substring(0, n-2) + s.substring(i);
        } else {
            return s.substring(0, n);
        }
    }

    public static String md5(String input) {
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            md.reset();
            md.update(input.getBytes("UTF-8"));
            byte[] digest = md.digest();
            String hash = new BigInteger(1, digest).toString(16);
            while(hash.length() < 32 ){
              hash = "0"+hash;
            }
            return hash;
        } catch (UnsupportedEncodingException ex) {
            Logger.getLogger(Utils.class.getName()).log(Level.SEVERE, null, ex);
            return null;
        } catch (NoSuchAlgorithmException ex) {
            Logger.getLogger(Utils.class.getName()).log(Level.SEVERE, null, ex);
            return null;
        }
    }

    public static long longHashCode(String s) {
        long hash = 0;
        for (int i = 0; i < s.length(); i++) {
            hash = s.charAt(i) + hash * 31;
        }
        return hash;
    }

    public static String cleanupString(String str, int maxLen) {
        str = (str.length() > maxLen) ? str.substring(0, maxLen - 3) + "..." : str;
        str = str.replaceAll("\\s+", " ");
        return str;
    }

    public static String escapeWhitespace(String s) {
    	return new String(escapeWhitespace(s.getBytes()));
    }
    
    public static byte[] escapeWhitespace(byte[] in) {
    	int escapes = 0;
    	for (int i = 0; i < in.length; i++) {
    		byte c = in[i];
    		if (c == '\\' || c == '\n' || c == '\r' || c == '\t') {
    			escapes++;
    		}
    	}
    	
    	byte[] out = new byte[in.length + escapes];
    	int j = 0;
    	
    	for (int i = 0; i < in.length; i++) {
            byte ch = in[i];
            switch (ch) {
            case '\\':
            	out[j++] = '\\';
            	out[j++] = '\\';
                break;
            case '\n':
            	out[j++] = '\\';
            	out[j++] = 'n';
                break;
            case '\r':
            	out[j++] = '\\';
            	out[j++] = 'r';
                break;
            case '\t':
            	out[j++] = '\\';
            	out[j++] = 't';
                break;
            default:
            	out[j++] = ch;
            }
        }
    	return out;
    }
    
    public static String unescapeWhitespace(String s) {
    	byte[] bytes = s.getBytes();
    	int l = unescapeInPlace(bytes, bytes.length);
    	return new String(bytes, 0, l);
    }

	public static byte[] unescape(byte [] escaped, int length) {
		int newLength = 0;
		for (int i = 0; i < length; i++) {
			newLength++;
			if (escaped[i] == '\\') {
				i++;
			}
		}
		byte [] unescaped = new byte[newLength];
		int j = 0;	// index into unescaped string
		for (int i = 0; i < length; i++) {
			byte b = escaped[i];
			if (escaped[i] == '\\') {
				switch (escaped[i+1]) {
				case 'r': b = '\r'; break;
				case '\\': b = '\\'; break;
				case 'n': b = '\n'; break;
				case 't': b = '\t'; break;
	                                
				default:
					throw new RuntimeException("unexpected character following escape " + escaped[i+1]);
				}
				i++;
			}
			unescaped[j++] = b;
		}
		return unescaped;
	}

	public static int unescapeInPlace(byte [] escaped, int length) {
	    return Utils.unescapeInPlace(escaped, 0, length);
	}

	public static int unescapeInPlace(byte [] escaped, int offset, int length) {
		int j = 0;	// index into unescaped string
		for (int i = offset; i < length; i++) {
			byte b = escaped[i];
			if (escaped[i] == '\\') {
				switch (escaped[i+1]) {
				case 'r': b = '\r'; break;
				case '\\': b = '\\'; break;
				case 'n': b = '\n'; break;
				case 't': b = '\t'; break;
				default:
					throw new RuntimeException("unexpected character following escape " + escaped[i+1]);
				}
				i++;
			}
			escaped[j++] = b;
		}
		return j;
	}

	/**
	 * Convert a map-reduce key such as "324242.xml.7z" to 324242
	 * @param key
	 * @return
	 */
	static public long keyToId(Text key) {
	    String s = key.toString();
	    int i = s.indexOf('.');
	    if (i >= 0) {
	        return Long.valueOf(s.substring(0, i));
	    } else {
	        return -1;
	    }
	}

}
